From 31f77604f318e0fdb0a8f74622a2488c3f674307 Mon Sep 17 00:00:00 2001 From: "iap10@tetris.cl.cam.ac.uk" Date: Mon, 21 Mar 2005 09:00:17 +0000 Subject: [PATCH] bitkeeper revision 1.1236.1.99 (423e8d21lC6p0xGxw7U1ExVLXpZQag) Upgrade FreeBSD sparse tree from testing.bk to unstable.bk Signed-off-by: ian.pratt@cl.cam.ac.uk --- .../i386-xen/i386-xen/clock.c | 2 + .../i386-xen/i386-xen/ctrl_if.c | 171 ++++++++---- .../i386-xen/i386-xen/evtchn.c | 17 +- .../i386-xen/i386-xen/locore.s | 2 +- .../i386-xen/i386-xen/machdep.c | 107 ++++--- .../i386-xen/i386-xen/pmap.c | 3 + .../i386-xen/i386-xen/vm_machdep.c | 8 +- .../i386-xen/i386-xen/xen_machdep.c | 264 +++++++++--------- .../i386-xen/include/evtchn.h | 22 +- .../i386-xen/include/pmap.h | 4 +- .../i386-xen/include/vmparam.h | 2 +- .../i386-xen/include/xen-os.h | 111 +++++--- .../i386-xen/xen/blkfront/xb_blkfront.c | 136 +++++---- .../i386-xen/xen/misc/evtchn_dev.c | 18 +- .../i386-xen/xen/netfront/xn_netfront.c | 13 +- 15 files changed, 489 insertions(+), 391 deletions(-) diff --git a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/clock.c b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/clock.c index 393e091986..1a25569d4b 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/clock.c +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/clock.c @@ -105,6 +105,8 @@ int statclock_disable; #define TIMER_FREQ 1193182 #endif u_int timer_freq = TIMER_FREQ; +struct mtx clock_lock; + static const u_char daysinmonth[] = {31,28,31,30,31,30,31,31,30,31,30,31}; diff --git a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/ctrl_if.c b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/ctrl_if.c index 8e8ce9fde7..6c2f7df501 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/ctrl_if.c +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/ctrl_if.c @@ -38,6 +38,18 @@ #include #include +/* + * Extra ring macros to sync a consumer index up to the public producer index. + * Generally UNSAFE, but we use it for recovery and shutdown in some cases. + */ +#define RING_DROP_PENDING_REQUESTS(_r) \ + do { \ + (_r)->req_cons = (_r)->sring->req_prod; \ + } while (0) +#define RING_DROP_PENDING_RESPONSES(_r) \ + do { \ + (_r)->rsp_cons = (_r)->sring->rsp_prod; \ + } while (0) /* * Only used by initial domain which must create its own control-interface * event channel. This value is picked up by the user-space domain controller @@ -51,8 +63,8 @@ static struct mtx ctrl_if_lock; static int * ctrl_if_wchan = &ctrl_if_evtchn; -static CONTROL_RING_IDX ctrl_if_tx_resp_cons; -static CONTROL_RING_IDX ctrl_if_rx_req_cons; +static ctrl_front_ring_t ctrl_if_tx_ring; +static ctrl_back_ring_t ctrl_if_rx_ring; /* Incoming message requests. */ /* Primary message type -> message handler. */ @@ -85,7 +97,7 @@ TASKQUEUE_DECLARE(ctrl_if_txB); TASKQUEUE_DEFINE(ctrl_if_txB, NULL, NULL, {}); struct taskqueue **taskqueue_ctrl_if_tx[2] = { &taskqueue_ctrl_if_txA, &taskqueue_ctrl_if_txB }; -int ctrl_if_idx; +static int ctrl_if_idx = 0; static struct task ctrl_if_rx_tasklet; static struct task ctrl_if_tx_tasklet; @@ -95,8 +107,6 @@ static struct task ctrl_if_rxmsg_deferred_task; #define get_ctrl_if() ((control_if_t *)((char *)HYPERVISOR_shared_info + 2048)) -#define TX_FULL(_c) \ - (((_c)->tx_req_prod - ctrl_if_tx_resp_cons) == CONTROL_RING_SIZE) static void ctrl_if_notify_controller(void) @@ -114,13 +124,17 @@ ctrl_if_rxmsg_default_handler(ctrl_msg_t *msg, unsigned long id) static void __ctrl_if_tx_tasklet(void *context __unused, int pending __unused) { - control_if_t *ctrl_if = get_ctrl_if(); ctrl_msg_t *msg; - int was_full = TX_FULL(ctrl_if); + int was_full = RING_FULL(&ctrl_if_tx_ring); + RING_IDX i, rp; + + i = ctrl_if_tx_ring.rsp_cons; + rp = ctrl_if_tx_ring.sring->rsp_prod; + rmb(); /* Ensure we see all requests up to 'rp'. */ - while ( ctrl_if_tx_resp_cons != ctrl_if->tx_resp_prod ) + for ( ; i != rp; i++ ) { - msg = &ctrl_if->tx_ring[MASK_CONTROL_IDX(ctrl_if_tx_resp_cons)]; + msg = RING_GET_RESPONSE(&ctrl_if_tx_ring, i); /* Execute the callback handler, if one was specified. */ if ( msg->id != 0xFF ) @@ -131,77 +145,102 @@ __ctrl_if_tx_tasklet(void *context __unused, int pending __unused) ctrl_if_txmsg_id_mapping[msg->id].fn = NULL; } - /* - * Step over the message in the ring /after/ finishing reading it. As - * soon as the index is updated then the message may get blown away. - */ - smp_mb(); - ctrl_if_tx_resp_cons++; } - if ( was_full && !TX_FULL(ctrl_if) ) + /* + * Step over the message in the ring /after/ finishing reading it. As + * soon as the index is updated then the message may get blown away. + */ + smp_mb(); + ctrl_if_tx_ring.rsp_cons = i; + + if ( was_full && !RING_FULL(&ctrl_if_tx_ring) ) { wakeup(ctrl_if_wchan); /* bump idx so future enqueues will occur on the next taskq * process any currently pending tasks */ - ctrl_if_idx++; + ctrl_if_idx++; taskqueue_run(*taskqueue_ctrl_if_tx[(ctrl_if_idx-1) & 1]); } + } static void __ctrl_if_rxmsg_deferred_task(void *context __unused, int pending __unused) { ctrl_msg_t *msg; + CONTROL_RING_IDX dp; - while ( ctrl_if_rxmsg_deferred_cons != ctrl_if_rxmsg_deferred_prod ) + dp = ctrl_if_rxmsg_deferred_prod; + rmb(); /* Ensure we see all deferred requests up to 'dp'. */ + + while ( ctrl_if_rxmsg_deferred_cons != dp ) { msg = &ctrl_if_rxmsg_deferred[MASK_CONTROL_IDX( ctrl_if_rxmsg_deferred_cons++)]; (*ctrl_if_rxmsg_handler[msg->type])(msg, 0); } + } static void __ctrl_if_rx_tasklet(void *context __unused, int pending __unused) { - control_if_t *ctrl_if = get_ctrl_if(); ctrl_msg_t msg, *pmsg; + CONTROL_RING_IDX dp; + RING_IDX rp, i; + + i = ctrl_if_rx_ring.req_cons; + rp = ctrl_if_rx_ring.sring->req_prod; + dp = ctrl_if_rxmsg_deferred_prod; - while ( ctrl_if_rx_req_cons != ctrl_if->rx_req_prod ) + rmb(); /* Ensure we see all requests up to 'rp'. */ + + for ( ; i != rp; i++) { - pmsg = &ctrl_if->rx_ring[MASK_CONTROL_IDX(ctrl_if_rx_req_cons++)]; + pmsg = RING_GET_REQUEST(&ctrl_if_rx_ring, i); memcpy(&msg, pmsg, offsetof(ctrl_msg_t, msg)); + + if ( msg.length > sizeof(msg.msg)) + msg.length = sizeof(msg.msg); if ( msg.length != 0 ) memcpy(msg.msg, pmsg->msg, msg.length); if ( test_bit(msg.type, &ctrl_if_rxmsg_blocking_context) ) { - pmsg = &ctrl_if_rxmsg_deferred[MASK_CONTROL_IDX( - ctrl_if_rxmsg_deferred_prod++)]; - memcpy(pmsg, &msg, offsetof(ctrl_msg_t, msg) + msg.length); - taskqueue_enqueue(taskqueue_thread, &ctrl_if_rxmsg_deferred_task); + memcpy(&ctrl_if_rxmsg_deferred[MASK_CONTROL_IDX(dp++)], + &msg, offsetof(ctrl_msg_t, msg) + msg.length); } else { (*ctrl_if_rxmsg_handler[msg.type])(&msg, 0); } } + ctrl_if_rx_ring.req_cons = i; + + if ( dp != ctrl_if_rxmsg_deferred_prod ) + { + wmb(); + ctrl_if_rxmsg_deferred_prod = dp; + taskqueue_enqueue(taskqueue_thread, &ctrl_if_rxmsg_deferred_task); + } + } static void ctrl_if_interrupt(void *ctrl_sc) /* (int irq, void *dev_id, struct pt_regs *regs) */ { - control_if_t *ctrl_if = get_ctrl_if(); - if ( ctrl_if_tx_resp_cons != ctrl_if->tx_resp_prod ) + + if ( RING_HAS_UNCONSUMED_RESPONSES(&ctrl_if_tx_ring) ) taskqueue_enqueue(taskqueue_swi, &ctrl_if_tx_tasklet); - if ( ctrl_if_rx_req_cons != ctrl_if->rx_req_prod ) + if ( RING_HAS_UNCONSUMED_REQUESTS(&ctrl_if_rx_ring) ) taskqueue_enqueue(taskqueue_swi, &ctrl_if_rx_tasklet); + } int @@ -210,13 +249,13 @@ ctrl_if_send_message_noblock( ctrl_msg_handler_t hnd, unsigned long id) { - control_if_t *ctrl_if = get_ctrl_if(); unsigned long flags; + ctrl_msg_t *dmsg; int i; mtx_lock_irqsave(&ctrl_if_lock, flags); - if ( TX_FULL(ctrl_if) ) + if ( RING_FULL(&ctrl_if_tx_ring) ) { mtx_unlock_irqrestore(&ctrl_if_lock, flags); return EAGAIN; @@ -232,10 +271,11 @@ ctrl_if_send_message_noblock( msg->id = i; } - memcpy(&ctrl_if->tx_ring[MASK_CONTROL_IDX(ctrl_if->tx_req_prod)], - msg, sizeof(*msg)); - wmb(); /* Write the message before letting the controller peek at it. */ - ctrl_if->tx_req_prod++; + dmsg = RING_GET_REQUEST(&ctrl_if_tx_ring, + ctrl_if_tx_ring.req_prod_pvt); + memcpy(dmsg, msg, sizeof(*msg)); + ctrl_if_tx_ring.req_prod_pvt++; + RING_PUSH_REQUESTS(&ctrl_if_tx_ring); mtx_unlock_irqrestore(&ctrl_if_lock, flags); @@ -252,34 +292,35 @@ ctrl_if_send_message_block( long wait_state) { int rc, sst = 0; - + /* Fast path. */ - if ( (rc = ctrl_if_send_message_noblock(msg, hnd, id)) != EAGAIN ) - return rc; - - + if ( (rc = ctrl_if_send_message_noblock(msg, hnd, id)) != EAGAIN ) + goto done; + for ( ; ; ) { if ( (rc = ctrl_if_send_message_noblock(msg, hnd, id)) != EAGAIN ) break; - if ( sst != 0) - return EINTR; + if ( sst != 0) { + rc = EINTR; + goto done; + } sst = tsleep(ctrl_if_wchan, PWAIT|PCATCH, "ctlrwt", 10); } - + done: + return rc; } int ctrl_if_enqueue_space_callback(struct task *task) { - control_if_t *ctrl_if = get_ctrl_if(); /* Fast path. */ - if ( !TX_FULL(ctrl_if) ) + if ( !RING_FULL(&ctrl_if_tx_ring) ) return 0; (void)taskqueue_enqueue(*taskqueue_ctrl_if_tx[(ctrl_if_idx & 1)], task); @@ -290,13 +331,12 @@ ctrl_if_enqueue_space_callback(struct task *task) * certainly return 'not full'. */ smp_mb(); - return TX_FULL(ctrl_if); + return RING_FULL(&ctrl_if_tx_ring); } void ctrl_if_send_response(ctrl_msg_t *msg) { - control_if_t *ctrl_if = get_ctrl_if(); unsigned long flags; ctrl_msg_t *dmsg; @@ -305,11 +345,14 @@ ctrl_if_send_response(ctrl_msg_t *msg) * In this situation we may have src==dst, so no copying is required. */ mtx_lock_irqsave(&ctrl_if_lock, flags); - dmsg = &ctrl_if->rx_ring[MASK_CONTROL_IDX(ctrl_if->rx_resp_prod)]; + dmsg = RING_GET_RESPONSE(&ctrl_if_rx_ring, + ctrl_if_rx_ring.rsp_prod_pvt); if ( dmsg != msg ) memcpy(dmsg, msg, sizeof(*msg)); - wmb(); /* Write the message before letting the controller peek at it. */ - ctrl_if->rx_resp_prod++; + + ctrl_if_rx_ring.rsp_prod_pvt++; + RING_PUSH_RESPONSES(&ctrl_if_rx_ring); + mtx_unlock_irqrestore(&ctrl_if_lock, flags); ctrl_if_notify_controller(); @@ -323,7 +366,7 @@ ctrl_if_register_receiver( { unsigned long _flags; int inuse; - + mtx_lock_irqsave(&ctrl_if_lock, _flags); inuse = (ctrl_if_rxmsg_handler[type] != ctrl_if_rxmsg_default_handler); @@ -344,7 +387,7 @@ ctrl_if_register_receiver( } mtx_unlock_irqrestore(&ctrl_if_lock, _flags); - + return !inuse; } @@ -382,6 +425,7 @@ ctrl_if_suspend(void) unbind_evtchn_from_irq(ctrl_if_evtchn); } +#if 0 /** Reset the control interface progress pointers. * Marks the queues empty if 'clear' non-zero. */ @@ -398,10 +442,13 @@ ctrl_if_reset(int clear) ctrl_if_rx_req_cons = ctrl_if->rx_resp_prod; } - +#endif void ctrl_if_resume(void) { + control_if_t *ctrl_if = get_ctrl_if(); + + TRACE_ENTER; if ( xen_start_info->flags & SIF_INITDOMAIN ) { /* @@ -421,7 +468,10 @@ ctrl_if_resume(void) initdom_ctrlif_domcontroller_port = op.u.bind_interdomain.port2; } - ctrl_if_reset(0); + + /* Sync up with shared indexes. */ + FRONT_RING_ATTACH(&ctrl_if_tx_ring, &ctrl_if->tx_ring); + BACK_RING_ATTACH(&ctrl_if_rx_ring, &ctrl_if->rx_ring); ctrl_if_evtchn = xen_start_info->domain_controller_evtchn; ctrl_if_irq = bind_evtchn_to_irq(ctrl_if_evtchn); @@ -433,17 +483,24 @@ ctrl_if_resume(void) */ intr_add_handler("ctrl-if", ctrl_if_irq, (driver_intr_t*)ctrl_if_interrupt, - NULL, INTR_TYPE_NET | INTR_MPSAFE, NULL); + NULL, INTR_TYPE_NET, NULL); + TRACE_EXIT; + /* XXX currently assuming not MPSAFE */ } static void ctrl_if_init(void *dummy __unused) { + control_if_t *ctrl_if = get_ctrl_if(); + int i; for ( i = 0; i < 256; i++ ) ctrl_if_rxmsg_handler[i] = ctrl_if_rxmsg_default_handler; + FRONT_RING_ATTACH(&ctrl_if_tx_ring, &ctrl_if->tx_ring); + BACK_RING_ATTACH(&ctrl_if_rx_ring, &ctrl_if->rx_ring); + mtx_init(&ctrl_if_lock, "ctrlif", NULL, MTX_SPIN | MTX_NOWITNESS); TASK_INIT(&ctrl_if_tx_tasklet, 0, __ctrl_if_tx_tasklet, NULL); @@ -452,7 +509,7 @@ ctrl_if_init(void *dummy __unused) TASK_INIT(&ctrl_if_rxmsg_deferred_task, 0, __ctrl_if_rxmsg_deferred_task, NULL); - ctrl_if_reset(1); + ctrl_if_resume(); } @@ -464,13 +521,13 @@ ctrl_if_init(void *dummy __unused) int ctrl_if_transmitter_empty(void) { - return (get_ctrl_if()->tx_req_prod == ctrl_if_tx_resp_cons); + return (ctrl_if_tx_ring.sring->req_prod == ctrl_if_tx_ring.rsp_cons); } void ctrl_if_discard_responses(void) { - ctrl_if_tx_resp_cons = get_ctrl_if()->tx_resp_prod; + RING_DROP_PENDING_RESPONSES(&ctrl_if_tx_ring); } SYSINIT(ctrl_if_init, SI_SUB_DRIVERS, SI_ORDER_FIRST, ctrl_if_init, NULL); diff --git a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/evtchn.c b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/evtchn.c index 635a3bfe4e..c0b393d8dd 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/evtchn.c +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/evtchn.c @@ -54,9 +54,10 @@ evtchn_do_upcall(struct intrframe *frame) { unsigned long l1, l2; unsigned int l1i, l2i, port; - int irq, owned; + int irq; unsigned long flags; shared_info_t *s = HYPERVISOR_shared_info; + vcpu_info_t *vcpu_info = &s->vcpu_data[smp_processor_id()]; local_irq_save(flags); @@ -64,7 +65,7 @@ evtchn_do_upcall(struct intrframe *frame) { s->vcpu_data[0].evtchn_upcall_pending = 0; /* NB. No need for a barrier here -- XCHG is a barrier on x86. */ - l1 = xen_xchg(&s->evtchn_pending_sel, 0); + l1 = xen_xchg(&vcpu_info->evtchn_pending_sel, 0); while ( (l1i = ffs(l1)) != 0 ) { l1i--; @@ -77,17 +78,12 @@ evtchn_do_upcall(struct intrframe *frame) l2 &= ~(1 << l2i); port = (l1i << 5) + l2i; - if ((owned = mtx_owned(&sched_lock)) != 0) - mtx_unlock_spin_flags(&sched_lock, MTX_QUIET); if ( (irq = evtchn_to_irq[port]) != -1 ) { struct intsrc *isrc = intr_lookup_source(irq); intr_execute_handlers(isrc, frame); - } else { evtchn_device_upcall(port); } - if (owned) - mtx_lock_spin_flags(&sched_lock, MTX_QUIET); } } } @@ -451,12 +447,12 @@ static struct hw_interrupt_type pirq_type = { }; #endif - +#if 0 static void misdirect_interrupt(void *sc) { } - +#endif void irq_suspend(void) { int virq, irq, evtchn; @@ -572,9 +568,12 @@ evtchn_init(void *dummy __unused) } #endif +#if 0 (void) intr_add_handler("xb_mis", bind_virq_to_irq(VIRQ_MISDIRECT), (driver_intr_t *)misdirect_interrupt, NULL, INTR_TYPE_MISC, NULL); + +#endif } SYSINIT(evtchn_init, SI_SUB_INTR, SI_ORDER_ANY, evtchn_init, NULL); diff --git a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/locore.s b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/locore.s index 5146169162..427af5e628 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/locore.s +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/locore.s @@ -56,7 +56,7 @@ #include "assym.s" .section __xen_guest - .asciz "LOADER=generic,GUEST_VER=5.2.1,XEN_VER=2.0,BSD_SYMTAB" + .asciz "LOADER=generic,GUEST_VER=5.3,XEN_VER=3.0,BSD_SYMTAB" /* diff --git a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/machdep.c b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/machdep.c index ea813b897c..03f24fc170 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/machdep.c +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/machdep.c @@ -214,19 +214,7 @@ static struct trapframe proc0_tf; #ifndef SMP static struct pcpu __pcpu; #endif - -static void -map_range(void *physptr, unsigned long physptrindex, - unsigned long physindex, int count, unsigned int flags) { - int i; - unsigned long pte, ppa; - for (i = 0; i < count; i++) { - pte = ((unsigned long)physptr) + (physptrindex << 2) + (i << 2); - ppa = (PTOM(physindex + i) << PAGE_SHIFT) | flags | PG_V | PG_A; - xpq_queue_pt_update((pt_entry_t *)pte, ppa); - } - mcl_flush_queue(); -} +struct mtx icu_lock; struct mem_range_softc mem_range_softc; @@ -1377,20 +1365,18 @@ getmemsize(void) pmap_bootstrap((init_first)<< PAGE_SHIFT, 0); for (i = 0; i < 10; i++) phys_avail[i] = 0; -#ifdef MAXMEM - if (MAXMEM/4 < Maxmem) - Maxmem = MAXMEM/4; -#endif physmem = Maxmem; avail_end = ptoa(Maxmem) - round_page(MSGBUF_SIZE); phys_avail[0] = init_first << PAGE_SHIFT; phys_avail[1] = avail_end; } -extern pt_entry_t *KPTphys; -extern int kernbase; +extern unsigned long cpu0prvpage; +extern unsigned long *SMPpt; pteinfo_t *pteinfo_list; unsigned long *xen_machine_phys = ((unsigned long *)VADDR(1008, 0)); +int preemptable; +int gdt_set; /* Linux infection */ #define PAGE_OFFSET KERNBASE @@ -1406,8 +1392,6 @@ initvalues(start_info_t *startinfo) xendebug_flags = 0xffffffff; /* pre-zero unused mapped pages */ bzero((char *)(KERNBASE + (tmpindex << PAGE_SHIFT)), (1024 - tmpindex)*PAGE_SIZE); - - KPTphys = (pt_entry_t *)xpmap_ptom(__pa(startinfo->pt_base + PAGE_SIZE)); IdlePTD = (pd_entry_t *)xpmap_ptom(__pa(startinfo->pt_base)); XENPRINTF("IdlePTD %p\n", IdlePTD); XENPRINTF("nr_pages: %ld shared_info: 0x%lx flags: 0x%lx pt_base: 0x%lx " @@ -1416,6 +1400,10 @@ initvalues(start_info_t *startinfo) xen_start_info->flags, xen_start_info->pt_base, xen_start_info->mod_start, xen_start_info->mod_len); + /* setup self-referential mapping first so vtomach will work */ + xpq_queue_pt_update(IdlePTD + PTDPTDI , (unsigned long)IdlePTD | + PG_V | PG_A); + mcl_flush_queue(); /* Map proc0's UPAGES */ proc0uarea = (struct user *)(KERNBASE + (tmpindex << PAGE_SHIFT)); tmpindex += UAREA_PAGES; @@ -1431,6 +1419,25 @@ initvalues(start_info_t *startinfo) /* allocate page for ldt */ ldt = (union descriptor *)(KERNBASE + (tmpindex << PAGE_SHIFT)); tmpindex++; +#ifdef SMP + /* allocate cpu0 private page */ + cpu0prvpage = (KERNBASE + (tmpindex << PAGE_SHIFT)); + tmpindex++; + + /* allocate SMP page table */ + SMPpt = (unsigned long *)(KERNBASE + (tmpindex << PAGE_SHIFT)); + + /* Map the private page into the SMP page table */ + SMPpt[0] = vtomach(cpu0prvpage) | PG_RW | PG_M | PG_V | PG_A; + + /* map SMP page table RO */ + PT_SET_MA(SMPpt, vtomach(SMPpt) & ~PG_RW, TRUE); + + /* put the page table into the pde */ + xpq_queue_pt_update(IdlePTD + MPPTDI, xpmap_ptom((tmpindex << PAGE_SHIFT))| PG_M | PG_RW | PG_V | PG_A); + + tmpindex++; +#endif #ifdef PMAP_DEBUG pteinfo_list = (pteinfo_t *)(KERNBASE + (tmpindex << PAGE_SHIFT)); @@ -1444,17 +1451,20 @@ initvalues(start_info_t *startinfo) PT_CLEAR(KERNBASE + (i << PAGE_SHIFT), TRUE); /* allocate remainder of NKPT pages */ - map_range(IdlePTD, KPTDI + 1, tmpindex, NKPT-1, PG_U | PG_M | PG_RW); + for (i = 0; i < NKPT-1; i++, tmpindex++) + xpq_queue_pt_update(IdlePTD + KPTDI + i + 1, xpmap_ptom((tmpindex << PAGE_SHIFT))| PG_M | PG_RW | PG_V | PG_A); tmpindex += NKPT-1; - map_range(IdlePTD, PTDPTDI, __pa(xen_start_info->pt_base) >> PAGE_SHIFT, 1, 0); - xpq_queue_pt_update(KPTphys + tmpindex, xen_start_info->shared_info | PG_A | PG_V | PG_RW); + + + tmpindex += NKPT-1; + PT_UPDATES_FLUSH(); + HYPERVISOR_shared_info = (shared_info_t *)(KERNBASE + (tmpindex << PAGE_SHIFT)); + PT_SET_MA(HYPERVISOR_shared_info, xen_start_info->shared_info | PG_A | PG_V | PG_RW | PG_M, TRUE); tmpindex++; - mcl_flush_queue(); HYPERVISOR_shared_info->arch.pfn_to_mfn_frame_list = (unsigned long)xen_phys_machine; - HYPERVISOR_shared_info->arch.mfn_to_pfn_start = (unsigned long)xen_machine_phys; init_first = tmpindex; @@ -1465,6 +1475,7 @@ init386(void) { int gsel_tss, metadata_missing, off, x, error; struct pcpu *pc; + unsigned long gdtmachpfn; trap_info_t trap_table[] = { { 0, 0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(div)}, { 1, 0, GSEL(GCODE_SEL, SEL_KPL), (unsigned long) &IDTVEC(dbg)}, @@ -1541,6 +1552,9 @@ init386(void) gdt_segs[GDATA_SEL].ssd_limit = atop(0 - ((1 << 26) - (1 << 22) + (1 << 16))); #endif #ifdef SMP + /* this correspond to the cpu private page as mapped into the SMP page + * table in initvalues + */ pc = &SMP_prvspace[0].pcpu; gdt_segs[GPRIV_SEL].ssd_limit = atop(sizeof(struct privatespace) - 1); @@ -1553,17 +1567,15 @@ init386(void) gdt_segs[GPROC0_SEL].ssd_base = (int) &pc->pc_common_tss; for (x = 0; x < NGDT; x++) ssdtosd(&gdt_segs[x], &gdt[x].sd); - /* re-map GDT read-only */ - { - unsigned long gdtindex = (((unsigned long)gdt - KERNBASE) >> PAGE_SHIFT); - unsigned long gdtphys = PTOM(gdtindex); - map_range(KPTphys, gdtindex, gdtindex, 1, 0); - mcl_flush_queue(); - if (HYPERVISOR_set_gdt(&gdtphys, LAST_RESERVED_GDT_ENTRY + 1)) { - panic("set_gdt failed\n"); - } - lgdt_finish(); + + PT_SET_MA(gdt, *vtopte((unsigned long)gdt) & ~PG_RW, TRUE); + gdtmachpfn = vtomach(gdt) >> PAGE_SHIFT; + if (HYPERVISOR_set_gdt(&gdtmachpfn, LAST_RESERVED_GDT_ENTRY + 1)) { + XENPRINTF("set_gdt failed\n"); + } + lgdt_finish(); + gdt_set = 1; if ((error = HYPERVISOR_set_trap_table(trap_table)) != 0) { panic("set_trap_table failed - error %d\n", error); @@ -1580,7 +1592,6 @@ init386(void) PCPU_SET(prvspace, pc); PCPU_SET(curthread, &thread0); PCPU_SET(curpcb, thread0.td_pcb); - PCPU_SET(trap_nesting, 0); PCPU_SET(pdir, (unsigned long)IdlePTD); /* * Initialize mutexes. @@ -1588,6 +1599,11 @@ init386(void) */ mutex_init(); + mtx_init(&clock_lock, "clk", NULL, MTX_SPIN); + mtx_init(&icu_lock, "icu", NULL, MTX_SPIN | MTX_NOWITNESS); + + + /* make ldt memory segments */ /* * XXX - VM_MAXUSER_ADDRESS is an end address, not a max. And it @@ -1600,14 +1616,11 @@ init386(void) default_proc_ldt.ldt_base = (caddr_t)ldt; default_proc_ldt.ldt_len = 6; _default_ldt = (int)&default_proc_ldt; - PCPU_SET(currentldt, _default_ldt); - { - unsigned long ldtindex = (((unsigned long)ldt - KERNBASE) >> PAGE_SHIFT); - map_range(KPTphys, ldtindex, ldtindex, 1, 0); - mcl_flush_queue(); - xen_set_ldt((unsigned long) ldt, (sizeof ldt_segs / sizeof ldt_segs[0])); - } - + PCPU_SET(currentldt, _default_ldt) + PT_SET_MA(ldt, *vtopte((unsigned long)ldt) & ~PG_RW, TRUE); + xen_set_ldt((unsigned long) ldt, (sizeof ldt_segs / sizeof ldt_segs[0])); + + /* * Initialize the console before we print anything out. */ @@ -1638,12 +1651,15 @@ init386(void) KSTACK_PAGES * PAGE_SIZE - sizeof(struct pcb) - 16); PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); +#if 0 private_tss = 0; PCPU_SET(tss_gdt, &gdt[GPROC0_SEL].sd); PCPU_SET(common_tssd, *PCPU_GET(tss_gdt)); PCPU_SET(common_tss.tss_ioopt, (sizeof (struct i386tss)) << 16); +#endif HYPERVISOR_stack_switch(GSEL(GDATA_SEL, SEL_KPL), PCPU_GET(common_tss.tss_esp0)); + dblfault_tss.tss_esp = dblfault_tss.tss_esp0 = dblfault_tss.tss_esp1 = dblfault_tss.tss_esp2 = (int)&dblfault_stack[sizeof(dblfault_stack)]; dblfault_tss.tss_ss = dblfault_tss.tss_ss0 = dblfault_tss.tss_ss1 = @@ -1667,7 +1683,6 @@ init386(void) PT_UPDATES_FLUSH(); /* safe to enable xen page queue locking */ - xpq_init(); msgbufinit(msgbufp, MSGBUF_SIZE); /* XXX KMM I don't think we need call gates */ diff --git a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/pmap.c b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/pmap.c index ee61e80ed9..9f2ea02f67 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/pmap.c +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/pmap.c @@ -642,6 +642,7 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va) mtx_unlock_spin(&smp_rv_mtx); else critical_exit(); + PT_UPDATES_FLUSH(); } void @@ -681,6 +682,7 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) mtx_unlock_spin(&smp_rv_mtx); else critical_exit(); + PT_UPDATES_FLUSH(); } void @@ -716,6 +718,7 @@ pmap_invalidate_all(pmap_t pmap) mtx_unlock_spin(&smp_rv_mtx); else critical_exit(); + PT_UPDATES_FLUSH(); } #else /* !SMP */ /* diff --git a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/vm_machdep.c b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/vm_machdep.c index cff67833f7..7f04666723 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/i386-xen/vm_machdep.c +++ b/freebsd-5.3-xen-sparse/i386-xen/i386-xen/vm_machdep.c @@ -94,12 +94,13 @@ __FBSDID("$FreeBSD: src/sys/i386/i386/vm_machdep.c,v 1.219 2003/11/17 18:22:24 a #endif #include - +#if 0 #ifdef SMP static void cpu_reset_proxy(void); static u_int cpu_reset_proxyid; static volatile u_int cpu_reset_proxy_active; #endif +#endif static void sf_buf_init(void *arg); SYSINIT(sock_sf, SI_SUB_MBUF, SI_ORDER_ANY, sf_buf_init, NULL) @@ -462,6 +463,7 @@ kvtop(void *addr) * Force reset the processor by invalidating the entire address space! */ +#if 0 #ifdef SMP static void cpu_reset_proxy() @@ -473,10 +475,10 @@ cpu_reset_proxy() stop_cpus((1<> PAGE_SHIFT); - mcl_queue[mcl_idx].args[1] = (unsigned long)ma; - mcl_queue[mcl_idx].args[2] = 0; - mcl_increment_idx(); - XPQ_UNLOCK(&update_lock, flags); + SET_VCPU(); + + MCL_QUEUE[MCL_IDX].op = __HYPERVISOR_update_va_mapping; + MCL_QUEUE[MCL_IDX].args[0] = (unsigned long)va; + MCL_QUEUE[MCL_IDX].args[1] = (unsigned long)ma; + MCL_QUEUE[MCL_IDX].args[2] = UVMF_INVLPG; + mcl_increment_idx(); } @@ -554,72 +549,63 @@ mcl_queue_pt_update(vm_offset_t va, vm_paddr_t ma) void xpq_queue_pt_switch(uint32_t val) { - unsigned long flags = 0; - vm_paddr_t ma = xpmap_ptom(val) & PG_FRAME; - - XPQ_LOCK(&update_lock, flags); - xpq_queue[xpq_idx].ptr = ma | MMU_EXTENDED_COMMAND; - xpq_queue[xpq_idx].val = MMUEXT_NEW_BASEPTR; - xpq_increment_idx(); - XPQ_UNLOCK(&update_lock, flags); + vm_paddr_t ma = xpmap_ptom(val) & PG_FRAME; + SET_VCPU(); + + XPQ_QUEUE[XPQ_IDX].ptr = ma | MMU_EXTENDED_COMMAND; + XPQ_QUEUE[XPQ_IDX].val = MMUEXT_NEW_BASEPTR; + xpq_increment_idx(); } void xpq_queue_pin_table(uint32_t pa, int type) { - unsigned long flags = 0; - XPQ_LOCK(&update_lock, flags); - xpq_queue[xpq_idx].ptr = pa | MMU_EXTENDED_COMMAND; - switch (type) { - case XPQ_PIN_L1_TABLE: - xpq_queue[xpq_idx].val = MMUEXT_PIN_L1_TABLE; - break; - case XPQ_PIN_L2_TABLE: - xpq_queue[xpq_idx].val = MMUEXT_PIN_L2_TABLE; - break; - } - xpq_increment_idx(); - XPQ_UNLOCK(&update_lock, flags); + SET_VCPU(); + + + XPQ_QUEUE[XPQ_IDX].ptr = pa | MMU_EXTENDED_COMMAND; + switch (type) { + case XPQ_PIN_L1_TABLE: + XPQ_QUEUE[XPQ_IDX].val = MMUEXT_PIN_L1_TABLE; + break; + case XPQ_PIN_L2_TABLE: + XPQ_QUEUE[XPQ_IDX].val = MMUEXT_PIN_L2_TABLE; + break; + } + xpq_increment_idx(); } void xpq_queue_unpin_table(uint32_t pa) { - unsigned long flags = 0; - - XPQ_LOCK(&update_lock, flags); - xpq_queue[xpq_idx].ptr = pa | MMU_EXTENDED_COMMAND; - xpq_queue[xpq_idx].val = MMUEXT_UNPIN_TABLE; - xpq_increment_idx(); - XPQ_UNLOCK(&update_lock, flags); + SET_VCPU(); + + XPQ_QUEUE[XPQ_IDX].ptr = pa | MMU_EXTENDED_COMMAND; + XPQ_QUEUE[XPQ_IDX].val = MMUEXT_UNPIN_TABLE; + xpq_increment_idx(); } void xpq_queue_set_ldt(vm_offset_t va, uint32_t entries) { - unsigned long flags = 0; - - XPQ_LOCK(&update_lock, flags); - KASSERT(va == (va & PG_FRAME), ("ldt not page aligned")); - xpq_queue[xpq_idx].ptr = MMU_EXTENDED_COMMAND | va; - xpq_queue[xpq_idx].val = MMUEXT_SET_LDT | - (entries << MMUEXT_CMD_SHIFT); - xpq_increment_idx(); - XPQ_UNLOCK(&update_lock, flags); + SET_VCPU(); + + KASSERT(va == (va & PG_FRAME), ("ldt not page aligned")); + XPQ_QUEUE[XPQ_IDX].ptr = MMU_EXTENDED_COMMAND | va; + XPQ_QUEUE[XPQ_IDX].val = MMUEXT_SET_LDT | + (entries << MMUEXT_CMD_SHIFT); + xpq_increment_idx(); } void xpq_queue_tlb_flush() { - unsigned long flags = 0; - - XPQ_LOCK(&update_lock, flags); - - xpq_queue[xpq_idx].ptr = MMU_EXTENDED_COMMAND; - xpq_queue[xpq_idx].val = MMUEXT_TLB_FLUSH; - xpq_increment_idx(); - XPQ_UNLOCK(&update_lock, flags); + SET_VCPU(); + + XPQ_QUEUE[XPQ_IDX].ptr = MMU_EXTENDED_COMMAND; + XPQ_QUEUE[XPQ_IDX].val = MMUEXT_TLB_FLUSH; + xpq_increment_idx(); } diff --git a/freebsd-5.3-xen-sparse/i386-xen/include/evtchn.h b/freebsd-5.3-xen-sparse/i386-xen/include/evtchn.h index 3e962e3014..22e960652d 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/include/evtchn.h +++ b/freebsd-5.3-xen-sparse/i386-xen/include/evtchn.h @@ -9,11 +9,28 @@ #ifndef __ASM_EVTCHN_H__ #define __ASM_EVTCHN_H__ - +#include #include #include #include +#ifdef SMP +#include /* XXX for time.h */ +#include /* XXX for pcpu.h */ +#include /* XXX for PCPU_GET */ +extern int gdt_set; +static inline int +smp_processor_id(void) +{ + if (likely(gdt_set)) + return PCPU_GET(cpuid); + return 0; +} + +#else +#define smp_processor_id() 0 +#endif + /* * LOW-LEVEL DEFINITIONS */ @@ -38,6 +55,7 @@ static inline void unmask_evtchn(int port) { shared_info_t *s = HYPERVISOR_shared_info; + vcpu_info_t *vcpu_info = &s->vcpu_data[smp_processor_id()]; synch_clear_bit(port, &s->evtchn_mask[0]); @@ -46,7 +64,7 @@ unmask_evtchn(int port) * a real IO-APIC we 'lose the interrupt edge' if the channel is masked. */ if ( synch_test_bit (port, &s->evtchn_pending[0]) && - !synch_test_and_set_bit(port>>5, &s->evtchn_pending_sel) ) + !synch_test_and_set_bit(port>>5, &vcpu_info->evtchn_pending_sel) ) { s->vcpu_data[0].evtchn_upcall_pending = 1; if ( !s->vcpu_data[0].evtchn_upcall_mask ) diff --git a/freebsd-5.3-xen-sparse/i386-xen/include/pmap.h b/freebsd-5.3-xen-sparse/i386-xen/include/pmap.h index 9e838b9bd4..a7596b0e91 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/include/pmap.h +++ b/freebsd-5.3-xen-sparse/i386-xen/include/pmap.h @@ -149,8 +149,8 @@ */ #ifdef SMP -#define MPPTDI (NPDEPTD-1) /* per cpu ptd entry */ -#define KPTDI (MPPTDI-NKPDE-XEN_PAGES /* start of kernel virtual pde's */ +#define MPPTDI (NPDEPTD-1-XEN_PAGES) /* per cpu ptd entry */ +#define KPTDI (MPPTDI-NKPDE) /* start of kernel virtual pde's */ #else #define KPTDI (NPDEPTD-NKPDE-XEN_PAGES) /* start of kernel virtual pde's */ #endif /* SMP */ diff --git a/freebsd-5.3-xen-sparse/i386-xen/include/vmparam.h b/freebsd-5.3-xen-sparse/i386-xen/include/vmparam.h index 7fa9af3c68..9315c606af 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/include/vmparam.h +++ b/freebsd-5.3-xen-sparse/i386-xen/include/vmparam.h @@ -105,7 +105,7 @@ #define UPT_MAX_ADDRESS VADDR(PTDPTDI, PTDPTDI) #define UPT_MIN_ADDRESS VADDR(PTDPTDI, 0) -#define VM_MAXUSER_ADDRESS VADDR(PTDPTDI-1, 0) +#define VM_MAXUSER_ADDRESS VADDR(PTDPTDI, 0) #define USRSTACK VM_MAXUSER_ADDRESS diff --git a/freebsd-5.3-xen-sparse/i386-xen/include/xen-os.h b/freebsd-5.3-xen-sparse/i386-xen/include/xen-os.h index e483fc535c..47d726a040 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/include/xen-os.h +++ b/freebsd-5.3-xen-sparse/i386-xen/include/xen-os.h @@ -6,6 +6,7 @@ #ifndef _OS_H_ #define _OS_H_ +#include #ifndef NULL #define NULL (void *)0 @@ -58,6 +59,11 @@ void printk(const char *fmt, ...); /* some function prototypes */ void trap_init(void); +extern int preemptable; +#define preempt_disable() (preemptable = 0) +#define preempt_enable() (preemptable = 1) +#define preempt_enable_no_resched() (preemptable = 1) + /* * STI/CLI equivalents. These basically set and clear the virtual @@ -68,70 +74,74 @@ void trap_init(void); #define likely(x) __builtin_expect((x),1) #define unlikely(x) __builtin_expect((x),0) -#define __cli() \ -do { \ - HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask = 1; \ - barrier(); \ + + +#define __cli() \ +do { \ + vcpu_info_t *_vcpu; \ + preempt_disable(); \ + _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ + _vcpu->evtchn_upcall_mask = 1; \ + preempt_enable_no_resched(); \ + barrier(); \ } while (0) -#define __sti() \ -do { \ - shared_info_t *_shared = HYPERVISOR_shared_info; \ - barrier(); \ - _shared->vcpu_data[0].evtchn_upcall_mask = 0; \ - barrier(); /* unmask then check (avoid races) */ \ - if ( unlikely(_shared->vcpu_data[0].evtchn_upcall_pending) ) \ - force_evtchn_callback(); \ +#define __sti() \ +do { \ + vcpu_info_t *_vcpu; \ + barrier(); \ + preempt_disable(); \ + _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ + _vcpu->evtchn_upcall_mask = 0; \ + barrier(); /* unmask then check (avoid races) */ \ + if ( unlikely(_vcpu->evtchn_upcall_pending) ) \ + force_evtchn_callback(); \ + preempt_enable(); \ } while (0) + #define __save_flags(x) \ do { \ - (x) = HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask; \ + vcpu_info_t *vcpu; \ + vcpu = HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ + (x) = _vcpu->evtchn_upcall_mask; \ } while (0) -#define __restore_flags(x) \ -do { \ - shared_info_t *_shared = HYPERVISOR_shared_info; \ - barrier(); \ - if ( (_shared->vcpu_data[0].evtchn_upcall_mask = (x)) == 0 ) { \ - barrier(); /* unmask then check (avoid races) */ \ - if ( unlikely(_shared->vcpu_data[0].evtchn_upcall_pending) ) \ - force_evtchn_callback(); \ - } \ +#define __restore_flags(x) \ +do { \ + vcpu_info_t *_vcpu; \ + barrier(); \ + preempt_disable(); \ + _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ + if ((_vcpu->evtchn_upcall_mask = (x)) == 0) { \ + barrier(); /* unmask then check (avoid races) */ \ + if ( unlikely(_vcpu->evtchn_upcall_pending) ) \ + force_evtchn_callback(); \ + preempt_enable(); \ + } else \ + preempt_enable_no_resched(); \ } while (0) -#define __save_and_cli(x) \ -do { \ - (x) = HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask; \ - HYPERVISOR_shared_info->vcpu_data[0].evtchn_upcall_mask = 1; \ - barrier(); \ -} while (0) -#define __save_and_sti(x) \ -do { \ - shared_info_t *_shared = HYPERVISOR_shared_info; \ - barrier(); \ - (x) = _shared->vcpu_data[0].evtchn_upcall_mask; \ - _shared->vcpu_data[0].evtchn_upcall_mask = 0; \ - barrier(); /* unmask then check (avoid races) */ \ - if ( unlikely(_shared->vcpu_data[0].evtchn_upcall_pending) ) \ - force_evtchn_callback(); \ +#define __save_and_cli(x) \ +do { \ + vcpu_info_t *_vcpu; \ + preempt_disable(); \ + _vcpu = &HYPERVISOR_shared_info->vcpu_data[smp_processor_id()]; \ + (x) = _vcpu->evtchn_upcall_mask; \ + _vcpu->evtchn_upcall_mask = 1; \ + preempt_enable_no_resched(); \ + barrier(); \ } while (0) -#ifdef SMP -/* extra macros need for the SMP case */ -#error "global_irq_* not defined" -#endif #define cli() __cli() #define sti() __sti() #define save_flags(x) __save_flags(x) #define restore_flags(x) __restore_flags(x) #define save_and_cli(x) __save_and_cli(x) -#define save_and_sti(x) __save_and_sti(x) #define local_irq_save(x) __save_and_cli(x) -#define local_irq_set(x) __save_and_sti(x) #define local_irq_restore(x) __restore_flags(x) #define local_irq_disable() __cli() #define local_irq_enable() __sti() @@ -141,9 +151,20 @@ do { \ #define mb() #define rmb() -#define smp_mb() #define wmb() - +#ifdef SMP +#define smp_mb() mb() +#define smp_rmb() rmb() +#define smp_wmb() wmb() +#define smp_read_barrier_depends() read_barrier_depends() +#define set_mb(var, value) do { xchg(&var, value); } while (0) +#else +#define smp_mb() barrier() +#define smp_rmb() barrier() +#define smp_wmb() barrier() +#define smp_read_barrier_depends() do { } while(0) +#define set_mb(var, value) do { var = value; barrier(); } while (0) +#endif /* This is a barrier for the compiler only, NOT the processor! */ diff --git a/freebsd-5.3-xen-sparse/i386-xen/xen/blkfront/xb_blkfront.c b/freebsd-5.3-xen-sparse/i386-xen/xen/blkfront/xb_blkfront.c index 66c80f3ece..d1117febf0 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/xen/blkfront/xb_blkfront.c +++ b/freebsd-5.3-xen-sparse/i386-xen/xen/blkfront/xb_blkfront.c @@ -100,8 +100,10 @@ static unsigned int blkif_irq; static int blkif_control_rsp_valid; static blkif_response_t blkif_control_rsp; -static unsigned long xb_rec_ring_free; -blkif_request_t xb_rec_ring[BLKIF_RING_SIZE]; /* shadow recovery ring */ +static blkif_front_ring_t blk_ring; + +static unsigned long rec_ring_free; +blkif_request_t rec_ring[RING_SIZE(&blk_ring)]; /* shadow recovery ring */ /* XXX move to xb_vbd.c when VBD update support is added */ #define MAX_VBDS 64 @@ -115,16 +117,10 @@ static unsigned int xb_kick_pending; static struct mtx blkif_io_lock; -static blkif_ring_t *xb_blk_ring; -static BLKIF_RING_IDX xb_resp_cons; /* Response consumer for comms ring. */ -static BLKIF_RING_IDX xb_req_prod; /* Private request producer */ static int xb_recovery = 0; /* "Recovery in progress" flag. Protected * by the blkif_io_lock */ -/* We plug the I/O ring if the driver is suspended or if the ring is full. */ -#define BLKIF_RING_FULL (((xb_req_prod - xb_resp_cons) == BLKIF_RING_SIZE) || \ - (blkif_state != BLKIF_STATE_CONNECTED)) void blkif_completion(blkif_request_t *req); void xb_response_intr(void *); @@ -135,13 +131,13 @@ void xb_response_intr(void *); static inline int GET_ID_FROM_FREELIST( void ) { - unsigned long free = xb_rec_ring_free; + unsigned long free = rec_ring_free; - KASSERT(free <= BLKIF_RING_SIZE, ("free %lu > BLKIF_RING_SIZE", free)); + KASSERT(free <= RING_SIZE(&blk_ring), ("free %lu > RING_SIZE", free)); - xb_rec_ring_free = xb_rec_ring[free].id; + rec_ring_free = rec_ring[free].id; - xb_rec_ring[free].id = 0x0fffffee; /* debug */ + rec_ring[free].id = 0x0fffffee; /* debug */ return free; } @@ -149,8 +145,8 @@ GET_ID_FROM_FREELIST( void ) static inline void ADD_ID_TO_FREELIST( unsigned long id ) { - xb_rec_ring[id].id = xb_rec_ring_free; - xb_rec_ring_free = id; + rec_ring[id].id = rec_ring_free; + rec_ring_free = id; } static inline void translate_req_to_pfn(blkif_request_t *xreq, @@ -188,7 +184,7 @@ static inline void translate_req_to_mfn(blkif_request_t *xreq, static inline void flush_requests(void) { - xb_blk_ring->req_prod = xb_req_prod; + RING_PUSH_REQUESTS(&blk_ring); notify_via_evtchn(blkif_evtchn); } @@ -207,12 +203,9 @@ xb_response_intr(void *xsc) struct xb_softc *sc = NULL; struct bio *bp; blkif_response_t *bret; - BLKIF_RING_IDX i, rp; + RING_IDX i, rp; unsigned long flags; - if (blkif_state == BLKIF_STATE_CLOSED) - return; - mtx_lock_irqsave(&blkif_io_lock, flags); if ( unlikely(blkif_state == BLKIF_STATE_CLOSED) || @@ -221,20 +214,20 @@ xb_response_intr(void *xsc) return; } - rp = xb_blk_ring->resp_prod; + rp = blk_ring.sring->rsp_prod; rmb(); /* Ensure we see queued responses up to 'rp'. */ /* sometimes we seem to lose i/o. stay in the interrupt handler while * there is stuff to process: continually recheck the response producer. */ - for ( i = xb_resp_cons; i != (rp = xb_blk_ring->resp_prod); i++ ) { + for ( i = blk_ring.rsp_cons; i != (rp = blk_ring.sring->rsp_prod); i++ ) { unsigned long id; - bret = &xb_blk_ring->ring[MASK_BLKIF_IDX(i)].resp; + bret = RING_GET_RESPONSE(&blk_ring, i); id = bret->id; - bp = (struct bio *)xb_rec_ring[id].id; + bp = (struct bio *)rec_ring[id].id; - blkif_completion(&xb_rec_ring[id]); + blkif_completion(&rec_ring[id]); ADD_ID_TO_FREELIST(id); /* overwrites req */ @@ -277,7 +270,7 @@ xb_response_intr(void *xsc) } } - xb_resp_cons = i; + blk_ring.rsp_cons = i; if (sc && xb_kick_pending) { xb_kick_pending = FALSE; @@ -323,8 +316,6 @@ xb_ioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td) { struct xb_softc *sc = (struct xb_softc *)dp->d_drv1; - TRACE_ENTER; - if (sc == NULL) return (ENXIO); @@ -355,8 +346,8 @@ xb_startio(struct xb_softc *sc) s = splbio(); for (bp = bioq_first(&sc->xb_bioq); - bp && !BLKIF_RING_FULL; - xb_req_prod++, queued++, bp = bioq_first(&sc->xb_bioq)) { + bp && !RING_FULL(&blk_ring); + blk_ring.req_prod_pvt++, queued++, bp = bioq_first(&sc->xb_bioq)) { /* Check if the buffer is properly aligned */ if ((vm_offset_t)bp->bio_data & PAGE_MASK) { @@ -388,9 +379,10 @@ xb_startio(struct xb_softc *sc) buffer_ma &= ~PAGE_MASK; /* Fill out a communications ring structure. */ - req = &xb_blk_ring->ring[MASK_BLKIF_IDX(xb_req_prod)].req; + req = RING_GET_REQUEST(&blk_ring, + blk_ring.req_prod_pvt); id = GET_ID_FROM_FREELIST(); - xb_rec_ring[id].id= (unsigned long)bp; + rec_ring[id].id= (unsigned long)bp; req->id = id; req->operation = (bp->bio_cmd == BIO_READ) ? BLKIF_OP_READ : @@ -409,11 +401,11 @@ xb_startio(struct xb_softc *sc) req->frame_and_sects[0] = buffer_ma | (fsect << 3) | lsect; /* Keep a private copy so we can reissue requests when recovering. */ - translate_req_to_pfn( &xb_rec_ring[id], req); + translate_req_to_pfn( &rec_ring[id], req); } - if (BLKIF_RING_FULL) + if (RING_FULL(&blk_ring)) xb_kick_pending = TRUE; if (queued != 0) @@ -503,8 +495,6 @@ xb_vbdinit(void) blkif_response_t rsp; vdisk_t *buf; - TRACE_ENTER; - buf = (vdisk_t *)malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK); /* Probe for disk information. */ @@ -538,28 +528,31 @@ void blkif_control_send(blkif_request_t *req, blkif_response_t *rsp) { unsigned long flags, id; + blkif_request_t *req_d; retry: - while ( (xb_req_prod - xb_resp_cons) == BLKIF_RING_SIZE ) { + while ( RING_FULL(&blk_ring) ) + { tsleep( req, PWAIT | PCATCH, "blkif", hz); } mtx_lock_irqsave(&blkif_io_lock, flags); - if ( (xb_req_prod - xb_resp_cons) == BLKIF_RING_SIZE ) + if ( RING_FULL(&blk_ring) ) { mtx_unlock_irqrestore(&blkif_io_lock, flags); goto retry; } - xb_blk_ring->ring[MASK_BLKIF_IDX(xb_req_prod)].req = *req; + req_d = RING_GET_REQUEST(&blk_ring, blk_ring.req_prod_pvt); + *req_d = *req; id = GET_ID_FROM_FREELIST(); - xb_blk_ring->ring[MASK_BLKIF_IDX(xb_req_prod)].req.id = id; - xb_rec_ring[id].id = (unsigned long) req; + req_d->id = id; + rec_ring[id].id = (unsigned long) req; - translate_req_to_pfn( &xb_rec_ring[id], req ); + translate_req_to_pfn( &rec_ring[id], req ); - xb_req_prod++; + blk_ring.req_prod_pvt++; flush_requests(); mtx_unlock_irqrestore(&blkif_io_lock, flags); @@ -602,7 +595,7 @@ blkif_send_interface_connect(void) blkif_fe_interface_connect_t *msg = (void*)cmsg.msg; msg->handle = 0; - msg->shmem_frame = (vtomach(xb_blk_ring) >> PAGE_SHIFT); + msg->shmem_frame = (vtomach(blk_ring.sring) >> PAGE_SHIFT); ctrl_if_send_message_block(&cmsg, NULL, 0, TASK_UNINTERRUPTIBLE); } @@ -622,9 +615,9 @@ blkif_free(void) mtx_unlock_irqrestore(&blkif_io_lock, flags); /* Free resources associated with old device channel. */ - if (xb_blk_ring) { - free(xb_blk_ring, M_DEVBUF); - xb_blk_ring = NULL; + if (blk_ring.sring != NULL) { + free(blk_ring.sring, M_DEVBUF); + blk_ring.sring = NULL; } /* free_irq(blkif_irq, NULL);*/ blkif_irq = 0; @@ -642,10 +635,10 @@ blkif_close(void) static void blkif_disconnect(void) { - if (xb_blk_ring) free(xb_blk_ring, M_DEVBUF); - xb_blk_ring = (blkif_ring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK); - xb_blk_ring->req_prod = xb_blk_ring->resp_prod = 0; - xb_resp_cons = xb_req_prod = 0; + if (blk_ring.sring) free(blk_ring.sring, M_DEVBUF); + blk_ring.sring = (blkif_sring_t *)malloc(PAGE_SIZE, M_DEVBUF, M_WAITOK); + SHARED_RING_INIT(blk_ring.sring); + FRONT_RING_INIT(&blk_ring, blk_ring.sring); blkif_state = BLKIF_STATE_DISCONNECTED; blkif_send_interface_connect(); } @@ -663,36 +656,39 @@ blkif_recover(void) { int i; + blkif_request_t *req; /* Hmm, requests might be re-ordered when we re-issue them. * This will need to be fixed once we have barriers */ /* Stage 1 : Find active and move to safety. */ - for ( i = 0; i < BLKIF_RING_SIZE; i++ ) { - if ( xb_rec_ring[i].id >= KERNBASE ) { - translate_req_to_mfn( - &xb_blk_ring->ring[xb_req_prod].req, &xb_rec_ring[i]); - xb_req_prod++; + for ( i = 0; i < RING_SIZE(&blk_ring); i++ ) { + if ( rec_ring[i].id >= KERNBASE ) { + req = RING_GET_REQUEST(&blk_ring, + blk_ring.req_prod_pvt); + translate_req_to_mfn(req, &rec_ring[i]); + blk_ring.req_prod_pvt++; } } - printk("blkfront: recovered %d descriptors\n",xb_req_prod); + printk("blkfront: recovered %d descriptors\n",blk_ring.req_prod_pvt); /* Stage 2 : Set up shadow list. */ - for ( i = 0; i < xb_req_prod; i++ ) { - xb_rec_ring[i].id = xb_blk_ring->ring[i].req.id; - xb_blk_ring->ring[i].req.id = i; - translate_req_to_pfn(&xb_rec_ring[i], &xb_blk_ring->ring[i].req); + for ( i = 0; i < blk_ring.req_prod_pvt; i++ ) { + req = RING_GET_REQUEST(&blk_ring, i); + rec_ring[i].id = req->id; + req->id = i; + translate_req_to_pfn(&rec_ring[i], req); } /* Stage 3 : Set up free list. */ - for ( ; i < BLKIF_RING_SIZE; i++ ){ - xb_rec_ring[i].id = i+1; + for ( ; i < RING_SIZE(&blk_ring); i++ ){ + rec_ring[i].id = i+1; } - xb_rec_ring_free = xb_req_prod; - xb_rec_ring[BLKIF_RING_SIZE-1].id = 0x0fffffff; + rec_ring_free = blk_ring.req_prod_pvt; + rec_ring[RING_SIZE(&blk_ring)-1].id = 0x0fffffff; - /* xb_blk_ring->req_prod will be set when we flush_requests().*/ + /* blk_ring.req_prod will be set when we flush_requests().*/ wmb(); /* Switch off recovery mode, using a memory barrier to ensure that @@ -877,11 +873,11 @@ xb_init(void *unused) printk("[XEN] Initialising virtual block device driver\n"); - xb_rec_ring_free = 0; - for (i = 0; i < BLKIF_RING_SIZE; i++) { - xb_rec_ring[i].id = i+1; + rec_ring_free = 0; + for (i = 0; i < RING_SIZE(&blk_ring); i++) { + rec_ring[i].id = i+1; } - xb_rec_ring[BLKIF_RING_SIZE-1].id = 0x0fffffff; + rec_ring[RING_SIZE(&blk_ring)-1].id = 0x0fffffff; (void)ctrl_if_register_receiver(CMSG_BLKIF_FE, blkif_ctrlif_rx, 0); @@ -921,5 +917,5 @@ blkif_completion(blkif_request_t *req) } } -MTX_SYSINIT(ioreq, &blkif_io_lock, "BIO LOCK", MTX_SPIN); +MTX_SYSINIT(ioreq, &blkif_io_lock, "BIO LOCK", MTX_SPIN | MTX_NOWITNESS); /* XXX how does one enroll a lock? */ SYSINIT(xbdev, SI_SUB_PSEUDO, SI_ORDER_ANY, xb_init, NULL) diff --git a/freebsd-5.3-xen-sparse/i386-xen/xen/misc/evtchn_dev.c b/freebsd-5.3-xen-sparse/i386-xen/xen/misc/evtchn_dev.c index de379b6bf9..fa06fb855e 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/xen/misc/evtchn_dev.c +++ b/freebsd-5.3-xen-sparse/i386-xen/xen/misc/evtchn_dev.c @@ -46,8 +46,10 @@ static devfs_handle_t xen_dev_dir; static unsigned long evtchn_dev_inuse; /* Notification ring, accessed via /dev/xen/evtchn. */ -#define RING_SIZE 2048 /* 2048 16-bit entries */ -#define RING_MASK(_i) ((_i)&(RING_SIZE-1)) + +#define EVTCHN_RING_SIZE 2048 /* 2048 16-bit entries */ + +#define EVTCHN_RING_MASK(_i) ((_i)&(EVTCHN_RING_SIZE-1)) static uint16_t *ring; static unsigned int ring_cons, ring_prod, ring_overflow; @@ -76,8 +78,8 @@ evtchn_device_upcall(int port) clear_evtchn(port); if ( ring != NULL ) { - if ( (ring_prod - ring_cons) < RING_SIZE ) { - ring[RING_MASK(ring_prod)] = (uint16_t)port; + if ( (ring_prod - ring_cons) < EVTCHN_RING_SIZE ) { + ring[EVTCHN_RING_MASK(ring_prod)] = (uint16_t)port; if ( ring_cons == ring_prod++ ) { wakeup(evtchn_waddr); } @@ -136,9 +138,9 @@ evtchn_read(struct cdev *dev, struct uio *uio, int ioflag) } /* Byte lengths of two chunks. Chunk split (if any) is at ring wrap. */ - if ( ((c ^ p) & RING_SIZE) != 0 ) { - bytes1 = (RING_SIZE - RING_MASK(c)) * sizeof(uint16_t); - bytes2 = RING_MASK(p) * sizeof(uint16_t); + if ( ((c ^ p) & EVTCHN_RING_SIZE) != 0 ) { + bytes1 = (EVTCHN_RING_SIZE - EVTCHN_RING_MASK(c)) * sizeof(uint16_t); + bytes2 = EVTCHN_RING_MASK(p) * sizeof(uint16_t); } else { bytes1 = (p - c) * sizeof(uint16_t); @@ -154,7 +156,7 @@ evtchn_read(struct cdev *dev, struct uio *uio, int ioflag) bytes2 = count - bytes1; } - if ( uiomove(&ring[RING_MASK(c)], bytes1, uio) || + if ( uiomove(&ring[EVTCHN_RING_MASK(c)], bytes1, uio) || ((bytes2 != 0) && uiomove(&ring[0], bytes2, uio))) /* keeping this around as its replacement is not equivalent * copyout(&ring[0], &buf[bytes1], bytes2) diff --git a/freebsd-5.3-xen-sparse/i386-xen/xen/netfront/xn_netfront.c b/freebsd-5.3-xen-sparse/i386-xen/xen/netfront/xn_netfront.c index e25f218eb3..23e762e304 100644 --- a/freebsd-5.3-xen-sparse/i386-xen/xen/netfront/xn_netfront.c +++ b/freebsd-5.3-xen-sparse/i386-xen/xen/netfront/xn_netfront.c @@ -264,7 +264,7 @@ static int netctrl_connected(void) { int ok; - + XENPRINTF("err %d up %d\n", netctrl.err, netctrl.up); if (netctrl.err) ok = netctrl.err; else if (netctrl.up == NETIF_DRIVER_STATUS_UP) @@ -424,8 +424,7 @@ xn_alloc_rx_buffers(struct xn_softc *sc) = INVALID_P2M_ENTRY; xn_rx_mcl[i].op = __HYPERVISOR_update_va_mapping; - xn_rx_mcl[i].args[0] = (unsigned long)mtod(m_new,vm_offset_t) - >> PAGE_SHIFT; + xn_rx_mcl[i].args[0] = (unsigned long)mtod(m_new,vm_offset_t); xn_rx_mcl[i].args[1] = 0; xn_rx_mcl[i].args[2] = 0; @@ -520,7 +519,7 @@ xn_rxeof(struct xn_softc *sc) mmu->val = (unsigned long)m->m_ext.ext_args >> PAGE_SHIFT; mmu++; mcl->op = __HYPERVISOR_update_va_mapping; - mcl->args[0] = (unsigned long)m->m_data >> PAGE_SHIFT; + mcl->args[0] = (unsigned long)m->m_data; mcl->args[1] = (rx->addr & ~PAGE_MASK) | PG_KERNEL; mcl->args[2] = 0; mcl++; @@ -1303,7 +1302,6 @@ netif_driver_status(netif_fe_driver_status_t *status) static void netif_ctrlif_rx(ctrl_msg_t *msg, unsigned long id) { - switch ( msg->subtype ) { case CMSG_NETIF_FE_INTERFACE_STATUS: @@ -1326,7 +1324,7 @@ netif_ctrlif_rx(ctrl_msg_t *msg, unsigned long id) break; } - ctrl_if_send_response(msg); + ctrl_if_send_response(msg); } #if 1 @@ -1338,7 +1336,6 @@ static int probe_interfaces(void) { int err = 0, conn = 0; int wait_i, wait_n = 100; - for ( wait_i = 0; wait_i < wait_n; wait_i++) { XENPRINTF("> wait_i=%d\n", wait_i); @@ -1421,7 +1418,7 @@ xn_init(void *unused) { int err = 0; - + netctrl_init(); (void)ctrl_if_register_receiver(CMSG_NETIF_FE, netif_ctrlif_rx, CALLBACK_IN_BLOCKING_CONTEXT); -- 2.30.2